{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MSD 歌曲推荐——推荐\n",
    "1. 定义不同模型中用户对item的打分函数\n",
    "2. 装载训练好的模型\n",
    "3. 根据模型，预测用户对item的打分\n",
    "4. 读取测试数据\n",
    "5. 根据用户对item的打分产生推荐，并计算推荐结果的评价指标"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# coding: utf-8\n",
    "# -*- coding:utf-8 -*-\n",
    "import sys\n",
    "reload(sys)\n",
    "sys.setdefaultencoding(\"utf-8\")\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import cPickle\n",
    "import scipy.io as sio\n",
    "\n",
    "#距离\n",
    "import scipy.spatial.distance as ssd\n",
    "\n",
    "from numpy.random import random  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. 对给定用户，推荐物品/计算打分\n",
    "不同的推荐算法，打分不同"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.1 基于用户的协同过滤"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### 预测用户uid对item iid的打分\n",
    "### similarity_matrix为用户与用户之间的相似性矩阵\n",
    "def user_CF_pred(uid, iid, similarity_matrix): \n",
    "    sim_accumulate=0.0  \n",
    "    rat_acc=0.0 \n",
    "    for user_id in item_users[iid]:  #对item iid打过分的所有用户\n",
    "        #计算当前用户与给item i打过分的用户之间的相似度\n",
    "        #sim = user_similarity(user_id, uid)\n",
    "        sim = similarity_matrix[user_id,uid]\n",
    "            \n",
    "        if sim != 0: \n",
    "            rat_acc += sim * (user_item_scores[user_id,iid] - users_mu[user_id])   #用户user对item i的打分\n",
    "            sim_accumulate += np.abs(sim)  \n",
    "        \n",
    "    if sim_accumulate != 0:   \n",
    "        score = users_mu[uid] + rat_acc/sim_accumulate\n",
    "    else: #no similar users,return average rates of the user\n",
    "        score = users_mu[uid]\n",
    "    \n",
    "    return score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.2 基于物品的协同过滤\n",
    "利用用户打过分的item中，与item最相似的n_Knns最物品计算预测打分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### 预测用户uid对item iid的打分\n",
    "### similarity_matrix为item与item之间的相似性矩阵\n",
    "### n_Knns最相似的物品的数目\n",
    "def item_CF_pred(uid, iid, similarity_matrix, n_Knns): \n",
    "    sim_accumulate=0.0  \n",
    "    rat_acc=0.0 \n",
    "    n_nn_items = 0\n",
    "    \n",
    "    #相似度排序\n",
    "    cur_items_similarity = np.array(similarity_matrix[iid,:])\n",
    "    cur_items_similarity = cur_items_similarity.flatten()\n",
    "    sort_index = sorted(((e,i) for i,e in enumerate(list(cur_items_similarity))), reverse=True)\n",
    "    \n",
    "    for i in range(0,len(sort_index)):\n",
    "        cur_item_index = sort_index[i][1]\n",
    "        \n",
    "        if n_nn_items >= n_Knns:  #相似的items已经足够多（>n_Knns）\n",
    "            break;\n",
    "        \n",
    "        if cur_item_index in user_items[uid]: #对用户打过分的item\n",
    "           #计算当前用户打过分item与其他item之间的相似度\n",
    "            #sim = item_similarity(cur_item_index, iid)\n",
    "            sim = similarity_matrix[iid, cur_item_index]\n",
    "            \n",
    "            if sim != 0: \n",
    "                rat_acc += sim * (user_item_scores[uid, cur_item_index])   #用户user对item i的打分\n",
    "                sim_accumulate += np.abs(sim)  \n",
    "        \n",
    "            n_nn_items += 1\n",
    "        \n",
    "    if sim_accumulate != 0:   \n",
    "        score = rat_acc/sim_accumulate\n",
    "    else:   #no similar items,return average rates of the user  \n",
    "        score = users_mu[uid]\n",
    "    \n",
    "    if score <0:\n",
    "        score = 0.0\n",
    "    \n",
    "    return score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.3 基于SVD的协同过滤"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def svd_CF_pred(uid, iid):  \n",
    "    score = mu + bi[iid] + bu[uid] + np.sum(qi[iid]* pu[uid])  \n",
    "    return score  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. 从文件读入训练好的模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# A method for loading data from JSON file\n",
    "#用于读取训练好的svd模型\n",
    "def load_json(filepath):\n",
    "    with open(filepath, 'r') as file:\n",
    "        dict_ = json.load(file)\n",
    "\n",
    "        mu = dict_['mu']\n",
    "        K = dict_['K']\n",
    "\n",
    "        bi = np.asarray(dict_['bi'])\n",
    "        bu = np.asarray(dict_['bu'])\n",
    "    \n",
    "        qi = np.asarray(dict_['qi'])\n",
    "        pu = np.asarray(dict_['pu'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "#用户和item的索引\n",
    "users_index = cPickle.load(open(\"users_index.pkl\", 'rb'))\n",
    "items_index = cPickle.load(open(\"items_index.pkl\", 'rb'))\n",
    "\n",
    "n_users = len(users_index)\n",
    "n_items = len(items_index)\n",
    "    \n",
    "#用户-物品关系矩阵R\n",
    "user_item_scores = sio.mmread(\"user_item_scores\").todense()\n",
    "    \n",
    "#倒排表\n",
    "##每个用户播放的歌曲\n",
    "user_items = cPickle.load(open(\"user_items.pkl\", 'rb'))\n",
    "##事件参加的用户\n",
    "item_users = cPickle.load(open(\"item_users.pkl\", 'rb'))\n",
    "\n",
    "#所有用户之间的相似度\n",
    "similarity_matrix_users = cPickle.load(open(\"users_similarity_playcount.pkl\", 'rb'))\n",
    "\n",
    "#所有item之间的相似度\n",
    "similarity_matrix_items = cPickle.load(open(\"items_similarity_playcount.pkl\", 'rb'))\n",
    "\n",
    "#svd模型\n",
    "load_json('svd_model.json')\n",
    "\n",
    "#每个用户的平均打分\n",
    "users_mu = cPickle.load(open(\"users_mu.pkl\", 'rb'))\n",
    "\n",
    "\n",
    "#训练好的svd模型\n",
    "#load_json('svd_model.json')\n",
    "with open('svd_model.json') as file:\n",
    "    dict_ = json.load(file)\n",
    "\n",
    "    mu = dict_['mu']\n",
    "    K = dict_['K']\n",
    "\n",
    "    bi = np.asarray(dict_['bi'])\n",
    "    bu = np.asarray(dict_['bu'])\n",
    "    \n",
    "    qi = np.asarray(dict_['qi'])\n",
    "    pu = np.asarray(dict_['pu'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. 根据模型，预测用户对item的打分\n",
    "不同的推荐算法，只是预测打分函数不同，\n",
    "user_items_scores[i] = user_CF_pred(cur_user_id, i)  #预测打分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#user：用户\n",
    "#返回推荐items及其打分（DataFrame）\n",
    "\n",
    "N_KNNS = 10\n",
    "\n",
    "def recommend(user):\n",
    "    cur_user_id = users_index[user]\n",
    "    \n",
    "    #训练集中该用户打过分的item\n",
    "    cur_user_items = user_items[cur_user_id]\n",
    "\n",
    "    #该用户对所有item的打分\n",
    "    user_items_scores = np.zeros(n_items)\n",
    "\n",
    "    #预测打分\n",
    "    for i in range(n_items):  # all items \n",
    "        if i not in cur_user_items: #训练集中没打过分\n",
    "            #user_items_scores[i] = user_CF_pred(cur_user_id, i, similarity_matrix_users)  #预测打分\n",
    "            #user_items_scores[i] = item_CF_pred(cur_user_id, i, similarity_matrix_items, N_KNNS)  #预测打分\n",
    "            user_items_scores[i] = svd_CF_pred(cur_user_id, i)  #预测打分\n",
    "    \n",
    "    #推荐\n",
    "    #Sort the indices of user_item_scores based upon their value，Also maintain the corresponding score\n",
    "    sort_index = sorted(((e,i) for i,e in enumerate(list(user_items_scores))), reverse=True)\n",
    "    \n",
    "    #Create a dataframe from the following\n",
    "    columns = ['item_id', 'score']\n",
    "    df = pd.DataFrame(columns=columns)\n",
    "         \n",
    "    #Fill the dataframe with top 20 (n_rec_items) item based recommendations\n",
    "    #sort_index = sort_index[0:n_rec_items]\n",
    "    #Fill the dataframe with all items based recommendations\n",
    "    for i in range(0,len(sort_index)):\n",
    "        cur_item_index = sort_index[i][1] \n",
    "        cur_item = list (items_index.keys()) [list (items_index.values()).index (cur_item_index)]\n",
    "            \n",
    "        if ~np.isnan(sort_index[i][0]) and cur_item_index not in cur_user_items:\n",
    "            df.loc[len(df)]=[cur_item, sort_index[i][0]]\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#recommend('b21e1b6b14b7b3b8b8e683e82ede0e59ad64e9f7')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  4. 读取测试数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dpath = './data/'\n",
    "df_triplet_test = pd.read_csv( dpath  + 'triplet_dataset_sub_test.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. 测试，并计算评价指标\n",
    "PR、覆盖度、RMSE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "#统计总的用户\n",
    "unique_users_test = df_triplet_test['user'].unique()\n",
    "\n",
    "#为每个用户推荐的item的数目\n",
    "N_RS_ITEMS = 10\n",
    "\n",
    "\n",
    "#性能评价参数初始化，用户计算Percison和Recall\n",
    "n_hits = 0\n",
    "n_total_rec_items = 0\n",
    "n_test_items = 0\n",
    "\n",
    "#所有被推荐商品的集合（对不同用户），用于计算覆盖度\n",
    "all_rec_items = set()\n",
    "\n",
    "#残差平方和，用与计算RMSE\n",
    "rss_test = 0.0\n",
    "\n",
    "#对每个测试用户\n",
    "for user in unique_users_test:\n",
    "    #测试集中该用户打过分的电影（用于计算评价指标的真实值）\n",
    "    if user not in users_index:   #user在训练集中没有出现过，新用户不能用协同过滤\n",
    "        print(str(user) + ' is a new user.\\n')\n",
    "        continue\n",
    "   \n",
    "    #测试集真实值\n",
    "    df_user_records_test= df_triplet_test[df_triplet_test.user == user]\n",
    "    \n",
    "    #对每个测试用户，计算该用户对训练集中未出现过的商品的打分，并基于该打分进行推荐（top N_RS_ITEMS）\n",
    "    #返回结果为DataFrame\n",
    "    df_rec_items = recommend(user)\n",
    "    for i in range(N_RS_ITEMS):\n",
    "        item = df_rec_items.iloc[i]['item_id']\n",
    "        \n",
    "        if item in df_user_records_test['song'].values:\n",
    "            n_hits += 1\n",
    "        all_rec_items.add(item)\n",
    "    \n",
    "    #计算rmse\n",
    "    #对测试集中的每条记录，计算真实值与预测之间的RMSE\n",
    "    for i in range(df_user_records_test.shape[0]):\n",
    "        item = df_user_records_test.iloc[i]['song']\n",
    "        score = df_user_records_test.iloc[i]['fractional_play_count']\n",
    "        \n",
    "        df1 = df_rec_items[df_rec_items.item_id == item]\n",
    "        if(df1.shape[0] == 0): #item不在推荐列表中，可能是新item在训练集中没有出现过，或者该用户已经打过分新item不能被协同过滤推荐\n",
    "            print(str(item) + ' is a new item or  user \\n')\n",
    "            continue\n",
    "        pred_score = df1['score'].values[0]\n",
    "        \n",
    "        rss_test += (pred_score - score)**2     #残差平方和\n",
    "    \n",
    "    #推荐的item总数\n",
    "    n_total_rec_items += N_RS_ITEMS\n",
    "    \n",
    "    #真实item的总数\n",
    "    n_test_items += df_user_records_test.shape[0]\n",
    "\n",
    "#Precision & Recall\n",
    "precision = n_hits / (1.0*n_total_rec_items)\n",
    "recall = n_hits / (1.0*n_test_items)\n",
    "\n",
    "#覆盖度：推荐商品占总需要推荐商品的比例\n",
    "coverage = len(all_rec_items) / (1.0* n_items)\n",
    "\n",
    "#打分的均方误差\n",
    "rmse=np.sqrt(rss_test / df_triplet_test.shape[0])  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.019425444596443228"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.018933333333333333"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.10375"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coverage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.054327201333121231"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rmse"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "           Precision       Recall         Coverage     RMSE\n",
    "user_CF    0.010534        0.010267       0.35375      0.050265\n",
    "item_CF    0.018878        0.0184         0.99375      0.051091\n",
    "svd_CF     0.019425        0.018933       0.10375      0.054327            "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
